home *** CD-ROM | disk | FTP | other *** search
/ Total Network Tools 2002 / NextStepPublishing-TotalNetworkTools2002-Win95.iso / Archive / Misc Servers / Zope.exe / GFDB0.PY < prev    next >
Encoding:
Python Source  |  1999-11-03  |  47.2 KB  |  1,413 lines
  1. """storage objects"""
  2.  
  3. verbosity = 0
  4.  
  5. import os
  6.  
  7. # use whatever kjbuckets sqlsem is using
  8. #from sqlsem import kjbuckets, maketuple
  9.  
  10. # error on checking of data integrity
  11. StorageError = "StorageError"
  12.  
  13. # use md5 checksum (stub if md5 unavailable?)
  14. def checksum(string):
  15.     from md5 import new
  16.     return new(string).digest()
  17.     
  18. def recursive_dump(data, prefix="["):
  19.     """for debugging"""
  20.     from types import StringType
  21.     if type(data) is StringType: 
  22.        #print prefix, data
  23.        return
  24.     p2 = prefix+"["
  25.     try:
  26.         for x in data:
  27.             recursive_dump(x, p2)
  28.     except:
  29.         print prefix, data
  30.         
  31. def checksum_dump(data, file):
  32.     """checksum and dump marshallable data to file"""
  33.     #print "checksum_dump", file
  34.     #recursive_dump(data)
  35.     from marshal import dumps, dump
  36.     #print "data\n",data
  37.     storage = dumps(data)
  38.     checkpair = (checksum(storage), storage)
  39.     dump(checkpair, file)
  40.  
  41. def checksum_undump(file):
  42.     """undump marshallable data from file, checksum"""
  43.     from marshal import load, loads
  44.     checkpair = load(file)
  45.     (check, storage) = checkpair
  46.     if checksum(storage)!=check:
  47.        raise StorageError, "data load checksum fails"
  48.     data = loads(storage)
  49.     return data
  50.     
  51. def backup_file(filename, backupname):
  52.     """backup file, if unopenable ignore"""
  53.     try:
  54.         f = open(filename, "rb")
  55.     except:
  56.         return
  57.     data = f.read()
  58.     f.close()
  59.     f = open(backupname, "wb")
  60.     f.write(data)
  61.     f.close()
  62.     
  63. def del_file(filename):
  64.     """delete file, ignore errors"""
  65.     from os import unlink
  66.     try:
  67.         unlink(filename)
  68.     except: 
  69.         pass
  70.  
  71. class Database0:
  72.    """quick and dirty in core database representation."""
  73.    
  74.    # db.log is not None == use db.log to log modifications
  75.    
  76.    # set for verbose prints
  77.    verbose = verbosity
  78.    
  79.    # set for read only copy
  80.    readonly = 0
  81.    
  82.    # set for temp/scratch db copy semantics
  83.    is_scratch = 0
  84.    
  85.    # set to add introspective tables
  86.    introspect = 1
  87.    
  88.    def __init__(self, shadowing=None, log=None):
  89.        """dictionary of relations."""
  90.        verbose = self.verbose
  91.        self.shadowing = shadowing
  92.        self.log = log
  93.        self.touched = 0
  94.        if log:
  95.            self.is_scratch = log.is_scratch
  96.        if shadowing and not log:
  97.            raise ValueError, "shadowing db requires log"
  98.        if verbose:
  99.            print "Database0 init"
  100.            if log:
  101.                log.verbose = 1
  102.        if shadowing:
  103.            # shadow structures of shadowed db
  104.            self.rels = shadow_dict(shadowing.rels, Relation0.unshadow)
  105.            self.datadefs = shadow_dict(shadowing.datadefs)
  106.            self.indices = shadow_dict(shadowing.indices)
  107.        else:
  108.            self.rels = {}
  109.            self.datadefs = {}
  110.            self.indices = {}
  111.            if self.introspect:
  112.                self.set_introspection()
  113.                
  114.    def set_introspection(self):
  115.        import gfintrospect
  116.        self["dual"] = gfintrospect.DualView()
  117.        self["__table_names__"] = gfintrospect.RelationsView()
  118.        self["__datadefs__"] = gfintrospect.DataDefsView()
  119.        self["__indices__"] = gfintrospect.IndicesView()
  120.        self["__columns__"] = gfintrospect.ColumnsView()
  121.        self["__indexcols__"] = gfintrospect.IndexAttsView()
  122.            
  123.    def reshadow(self, db, dblog):
  124.        """(re)make self into shadow of db with dblog"""
  125.        self.shadowing = db
  126.        self.log = dblog
  127.        self.rels = shadow_dict(db.rels, Relation0.unshadow)
  128.        self.datadefs = shadow_dict(db.datadefs)
  129.        self.indices = shadow_dict(db.indices)
  130.            
  131.    def clear(self):
  132.        """I'm not sure if database has circular structure, so this added"""
  133.        self.shadowing = None
  134.        self.log = None
  135.        self.rels = {}
  136.        self.datadefs = {}
  137.        self.indices = {}
  138.            
  139.    def commit(self):
  140.        """commit shadowed changes"""
  141.        verbose = self.verbose
  142.        if self.shadowing and self.touched:
  143.           # log commit handled elsewhere
  144.           #log = self.log
  145.           #if log and not log.is_scratch:
  146.               #if verbose: print "committing log"
  147.               #self.log.commit(verbose)
  148.           if verbose: print "committing rels"
  149.           self.rels.commit(verbose)
  150.           if verbose: print "committing datadefs"
  151.           self.datadefs.commit(verbose)
  152.           if verbose: print "committing indices"
  153.           self.indices.commit(verbose)
  154.           st = self.shadowing.touched
  155.           if not st:
  156.               if verbose: "print setting touched", self.touched
  157.               self.shadowing.touched = self.touched
  158.           elif verbose:
  159.               print "shadowed database is touched"
  160.        elif verbose:
  161.           print "db0: commit on nonshadow instance"
  162.        
  163.    def __setitem__(self, name, relation):
  164.        """bind a name (uppercased) to tuples as a relation."""
  165.        from string import upper
  166.        if self.indices.has_key(name):
  167.           raise NameError, "cannot set index"
  168.        self.rels[ upper(name) ] = relation
  169.        if self.verbose: print "db0 sets rel", name
  170.        
  171.    def add_index(self, name, index):
  172.        if self.rels.has_key(name):
  173.           raise NameError, `name`+": is relation"
  174.        self.indices[name] = index
  175.        if self.verbose: print "db0 sets index", name
  176.        
  177.    def drop_index(self, name):
  178.        if self.verbose: print "db0 drops index", name
  179.        del self.indices[name]
  180.        
  181.    def __getitem__(self, name):
  182.        if self.verbose: print "db0 gets rel", name
  183.        from string import upper
  184.        return self.rels[upper(name)]
  185.        
  186.    def get_for_update(self, name):
  187.        """note: does not imply updates, just possibility of them"""
  188.        verbose = self.verbose
  189.        if verbose: print "db0 gets rel for update", name
  190.        shadowing = self.shadowing
  191.        gotit = 0
  192.        from string import upper
  193.        name = upper(name)
  194.        rels = self.rels
  195.        if shadowing:
  196.            if rels.is_shadowed(name):
  197.                test = rels[name]
  198.                # do we really have a shadow or a db copy?
  199.                if test.is_shadow:
  200.                   gotit = 1
  201.            if not gotit:
  202.               if shadowing.has_relation(name):
  203.                  test = shadowing.get_for_update(name)
  204.               else:
  205.                  # uncommitted whole relation
  206.                  test = rels[name]
  207.                  gotit = 1
  208.        else:
  209.            test = rels[name]
  210.            gotit = 1
  211.        if self.readonly:
  212.            raise ValueError, "cannot update, db is read only"
  213.        elif test.is_view:
  214.            raise ValueError, "VIEW %s cannot be updated" % name
  215.        elif shadowing and not gotit:
  216.            if verbose: print "db0: making shadow for", name
  217.            if test.is_shadow: return test
  218.            shadow = Relation0(())
  219.            shadow = shadow.shadow(test, self.log, name, self)
  220.            rels[name] = shadow
  221.            return shadow
  222.        else:
  223.            return test
  224.        
  225.    def __delitem__(self, name):
  226.        if self.verbose: print "db0 drops rel", name
  227.        from string import upper
  228.        del self.rels[upper(name)]
  229.        
  230.    def relations(self):
  231.        return self.rels.keys()
  232.        
  233.    def has_relation(self, name):
  234.        return self.rels.has_key(name)
  235.        
  236.    def getdatadefs(self):
  237.        result = self.datadefs.values()
  238.        # sort to make create tables first, eg
  239.        result.sort()
  240.        return result
  241.        
  242.    def add_datadef(self, name, defn, logit=1):
  243.        """only log the datadef if logit is set, else ignore redefinitions"""
  244.        dd = self.datadefs
  245.        if logit and dd.has_key(name):
  246.           raise KeyError, `name`+": already defined"
  247.        if logit:
  248.           self.touched = 1
  249.        dd[name] = defn
  250.        
  251.    def has_datadef(self, name):
  252.        return self.datadefs.has_key(name)
  253.        
  254.    def drop_datadef(self, name):
  255.        if self.verbose: print "db0 drops datadef",name
  256.        dd = self.datadefs
  257.        #print dd.keys()
  258.        if not dd.has_key(name):
  259.           raise KeyError, `name`+": no such element"
  260.        del dd[name]
  261.        
  262.    def __repr__(self):
  263.        l = []
  264.        from string import join
  265.        l.append("INDICES: "+`self.indices.keys()`)
  266.        for (name, ddef) in self.datadefs.items():
  267.            l.append("data definition %s::\n%s" % (name, ddef))
  268.        for (name, rel) in self.rels.items():
  269.            l.append(name + ":")
  270.            l.append(rel.irepr())
  271.        return join(l, "\n\n")
  272.        
  273.    def bindings(self, fromlist):
  274.        """return (attdict, reldict, amb, ambatts) from fromlist = [(name,alias)...]
  275.           where reldict: alias > tuplelist
  276.                 attdict: attribute_name > unique_relation
  277.                 amb: dict of dottedname > (rel, att)
  278.                 ambatts: dict of ambiguous_name > witness_alias
  279.        """
  280.        from string import upper
  281.        rels = self.rels
  282.        ambiguous_atts = {}
  283.        ambiguous = {}
  284.        relseen = {}
  285.        attbindings = {}
  286.        relbindings = {}
  287.        for (name,alias) in fromlist:
  288.            name = upper(name)
  289.            alias = upper(alias)
  290.            if relseen.has_key(alias):
  291.               raise NameError, `alias` + ": bound twice in from list"
  292.            relseen[alias]=alias
  293.            try:
  294.                therel = rels[name]
  295.            except KeyError:
  296.                raise NameError, `name` + " no such relation in DB"
  297.            relbindings[alias] = therel
  298.            for attname in therel.attributes():
  299.                if not ambiguous_atts.has_key(attname):
  300.                   if attbindings.has_key(attname):
  301.                      oldrel = attbindings[attname]
  302.                      oldbind = (oldrel, attname)
  303.                      ambiguous[ "%s.%s" % oldbind] = oldbind
  304.                      del attbindings[attname]
  305.                      ambiguous_atts[attname]=alias
  306.                      newbind = (alias, attname)
  307.                      ambiguous[ "%s.%s" % newbind ] = newbind
  308.                   else:
  309.                      attbindings[attname] = alias
  310.                else:
  311.                   newbind = (alias, attname)
  312.                   ambiguous[ "%s.%s" % newbind ] = newbind
  313.        return (attbindings, relbindings, ambiguous, ambiguous_atts)
  314.        
  315. class File_Storage0:
  316.    """quick and dirty file storage mechanism.
  317.         relation names in directory/dbname.gfd
  318.           contains a white separated list of relation names
  319.         relations in directory/relname.grl
  320.           contains sequence of marshalled tuples reps
  321.           prefixed by marshalled list of atts
  322.    """
  323.    
  324.    verbose = verbosity
  325.    
  326.    def __init__(self, dbname, directory):
  327.        """directory must exist."""
  328.        if self.verbose: print "fs0 init:", dbname, directory
  329.        self.dbname = dbname
  330.        self.directory = directory
  331.        self.relation_implementation = Relation0
  332.        self.recovery_mode = 0
  333.        
  334.    def load(self, parser=None, forscratch=0):
  335.        # if logfile is present, need to recover
  336.        # error condition: fail to load relation, ddf, but no log file!
  337.        logfile = self.logfilename()
  338.        blogfile = self.backup_logfilename()
  339.        verbose = self.verbose
  340.        if verbose: print "fs0 load, checking", logfile
  341.        try:
  342.            testlog = open(logfile, "rb")
  343.            if verbose: print "fs0: opened", testlog
  344.            testlog.close()
  345.            testlog = open(blogfile, "rb")
  346.            testlog.close()
  347.            testlog = None
  348.        except:
  349.            recovery_mode = self.recovery_mode = 0
  350.            if verbose: print "recovery not needed"
  351.        else:
  352.            recovery_mode = self.recovery_mode = 1
  353.            if verbose: print "FS0 RECOVERY MODE LOAD!"
  354.        resultdb = Database0()
  355.        resultdb.is_scratch = forscratch
  356.        commands = self.get_initstatements()
  357.        #commands = parser.DoParse1(initstatements)
  358.        for command in commands:
  359.            if verbose: print "fs0 evals", command
  360.            command.relbind(resultdb)
  361.            command.eval()
  362.        for name in resultdb.relations():
  363.            if verbose: print "fs0 loads rel", name
  364.            rel = resultdb[name]
  365.            if rel.is_view:
  366.               # don't need to load views
  367.               continue
  368.            rel.set_empty()
  369.            try:
  370.                data = self.get_relation(name)
  371.            except StorageError, detail:
  372.                raise StorageError, "load failure %s: %s" % (name, detail)
  373.            attsin = tuple(data.attributes())
  374.            attsout = tuple(rel.attributes())
  375.            if attsin!=attsout:
  376.                raise StorageError, "rel %s: atts %s don't match %s" % (
  377.                   name, attsin, attsout)
  378.            rel.add_tuples( data.rows() )
  379.            # in sync!
  380.            rel.touched = 0
  381.        # db in sync
  382.        resultdb.touched = 0
  383.        # do recovery, if needed
  384.        if recovery_mode:
  385.            if verbose: print "fs0 recovering from logfile", logfile
  386.            # restart the log file only if db is not scratch
  387.            restart = not forscratch
  388.            Log = DB_Logger(logfile, blogfile)
  389.            if verbose: Log.verbose=1
  390.            Log.recover(resultdb, restart)
  391.            # do a checkpoint
  392.            self.recovery_mode = 0
  393.            if restart and not forscratch:
  394.                Log.shutdown()
  395.                Log = None
  396.                del_file(logfile)
  397.                if verbose: print "FS0: dumping database"
  398.                self.dump(resultdb)
  399.                Log = resultdb.log = DB_Logger(logfile, blogfile)
  400.                Log.startup()
  401.        elif not forscratch:
  402.            Log = DB_Logger(logfile, blogfile)
  403.            Log.startup()
  404.            resultdb.log = Log
  405.        return resultdb
  406.        
  407.    def relfilename(self, name):
  408.        #return "%s/%s.grl" % (self.directory, name)
  409.        return os.path.join(self.directory, name+".grl")
  410.        
  411.    def backup_relfilename(self, name):
  412.        #return "%s/%s.brl" % (self.directory, name)
  413.        return os.path.join(self.directory, name+".brl")
  414.        
  415.    def relfile(self, name, mode="rb"):
  416.        if self.recovery_mode:
  417.            return self.getfile_fallback(
  418.         self.backup_relfilename(name), self.relfilename(name), mode)
  419.        else:
  420.            name = self.relfilename(name)
  421.            return open(name, mode)
  422.            
  423.    def getfile_fallback(self, first, second, mode):
  424.        try:
  425.            return open(first, mode)
  426.        except:
  427.            return open(second, mode)
  428.        
  429.    def get_relation(self, name):
  430.        f = self.relfile(name, "rb")
  431.        rel = self.relation_implementation(())
  432.        try:
  433.            rel.load(f)
  434.        except StorageError:
  435.            if self.recovery_mode:
  436.                f = open(self.relfilename(name), "rb")
  437.                rel.load(f)
  438.            else:
  439.                raise StorageError, \
  440.   "fs: could not unpack backup rel file or rel file in recovery mode: "+name
  441.        return rel
  442.        
  443.    def dbfilename(self):
  444.        #return "%s/%s.gfd" % (self.directory, self.dbname)
  445.        return os.path.join(self.directory, self.dbname+".gfd")
  446.        
  447.    def backup_dbfilename(self):
  448.        #return "%s/%s.bfd" % (self.directory, self.dbname)
  449.        return os.path.join(self.directory, self.dbname+".bfd")
  450.        
  451.    def logfilename(self):
  452.        #return "%s/%s.gfl" % (self.directory, self.dbname)
  453.        return os.path.join(self.directory, self.dbname+".gfl")
  454.        
  455.    def backup_logfilename(self):
  456.        #return "%s/%s.glb" % (self.directory, self.dbname)
  457.        return os.path.join(self.directory, self.dbname+".glb")
  458.        
  459.    def get_initstat_file(self, mode):
  460.        if self.recovery_mode:
  461.            return self.getfile_fallback(
  462.             self.backup_dbfilename(), self.dbfilename(), mode)
  463.        else:
  464.            return open(self.dbfilename(), mode)
  465.        
  466.    def get_initstatements(self):
  467.        f = self.get_initstat_file("rb")
  468.        if self.verbose:
  469.            print "init statement from file", f
  470.        try:
  471.            data = checksum_undump(f)
  472.        except StorageError:
  473.            if self.recovery_mode:
  474.                f = open(self.dbfilename, "rb")
  475.                data = checksum_undump(f)
  476.            else:
  477.                raise StorageError, \
  478.   "could not unpack ddf backup or ddf file in recovery mode: "+self.dbname
  479.        f.close()
  480.        from sqlsem import deserialize
  481.        stats = map(deserialize, data)
  482.        return stats
  483.        
  484.    def dump(self, db):
  485.        """perform a checkpoint (no active transactions!)"""
  486.        # db should be non-shadowing db
  487.        # first thing: back up the log
  488.        backup_file(self.logfilename(), self.backup_logfilename())
  489.        verbose = self.verbose
  490.        if verbose: print "fs0: checkpointing db"
  491.        if db.is_scratch or db.readonly:
  492.            # don't need to do anything.
  493.            if verbose: print "fs0: scratch or readonly, returning"
  494.            return
  495.        log = db.log
  496.        if log:
  497.            log.commit()
  498.            if verbose:
  499.                print "DEBUG LOG TRACE"
  500.                log.dump()
  501.            log.shutdown()
  502.        if db.touched:
  503.            if verbose: print "fs0: db touched, backing up ddf file"
  504.            backup_file(self.dbfilename(), 
  505.                        self.backup_dbfilename())
  506.        relations = db.relations()
  507.        for r in relations:
  508.            rel = db[r]
  509.            #print r
  510.            if rel.touched:
  511.                if verbose: print "fs0: backing up touched rel", r
  512.                backup_file(self.relfilename(r), 
  513.                            self.backup_relfilename(r))
  514.        for r in relations:
  515.            if verbose: print "fs0: dumping relations now"
  516.            self.dumprelation(r, db[r])
  517.        if verbose: print "fs0: dumping datadefs now"
  518.        self.dumpdatadefs(db)
  519.        # del of logfile signals successful commit.
  520.        if verbose: print "fs0: successful dump, deleting log file"
  521.        logfilename = self.logfilename()
  522.        blogfilename = self.backup_logfilename()
  523.        del_file(logfilename)
  524.        del_file(blogfilename)
  525.        if db.touched:
  526.            if verbose: print "fs0: deleting backup ddf file"
  527.            del_file(self.backup_dbfilename())
  528.            db.touched = 0
  529.        for r in relations:
  530.            rel = db[r]
  531.            if rel.touched:
  532.                if verbose: print "fs0: deleting rel backup", r
  533.                del_file(self.backup_relfilename(r))
  534.            rel.touched = 0
  535.        if verbose: print "fs0: restarting db log"
  536.        log = db.log = DB_Logger(logfilename, blogfilename)
  537.        log.startup()
  538.        if verbose: print "fs0: dump complete"
  539.        self.recovery_mode = 0
  540.        
  541.    def dumprelation(self, name, rel, force=0):
  542.        """set force to ignore the "touch" flag."""
  543.        # ignore self.backup_mode
  544.        if (force or rel.touched) and not rel.is_view:
  545.            fn = self.relfilename(name)
  546.            if self.verbose:
  547.                print "dumping touched rel", name, "to", fn
  548.            f = open(fn, "wb")
  549.            rel.dump(f)
  550.        
  551.    def dumpdatadefs(self, db, force=0):
  552.        """set force to ignore the touch flag"""
  553.        # ignore self.backup_mode
  554.        if not (force or db.touched): return
  555.        #from marshal import dump, dumps
  556.        fn = self.dbfilename()
  557.        f = open(fn, "wb")
  558.        datadefs = db.getdatadefs()
  559.        from sqlsem import serialize
  560.        datadefsd = map(serialize, datadefs)
  561.        #for (defn, ser) in map(None, datadefs, datadefsd):
  562.            #print defn
  563.            #print ser
  564.            #dumps(ser)  ### debug test
  565.        checksum_dump(datadefsd, f)
  566.        f.close()
  567.        
  568. class Relation0:
  569.    """quick and dirty in core relation representation.
  570.         self.tuples contains tuples or 0 if erased.
  571.       tuples must not move (to preserve indices)
  572.       unless indices regenerate.
  573.    """
  574.    
  575.    is_view = 0 # Relation0 is not a view
  576.    
  577.    def __init__(self, attribute_names, tuples=None, filter=None):
  578.        from sqlsem import kjbuckets
  579.        self.indices = kjbuckets.kjGraph()
  580.        self.index_list = []
  581.        self.attribute_names = attribute_names
  582.        if tuples is None:
  583.           tuples = []
  584.        self.filter = filter
  585.        self.set_empty()
  586.        self.add_tuples(tuples)
  587.        # indices map attname > indices containing att
  588.        # relation to shadow and log (if non-null)
  589.        self.log = None
  590.        self.name = None # anonymous by default
  591.        self.is_shadow = 0
  592.        self.touched = 0
  593.        
  594.    def shadow(self, otherrelation, log, name, inshadowdb):
  595.        """return structural replica of otherrelation (as self)
  596.        
  597.           for non-updatable relation (eg, view) may return otherrelation"""
  598.        if otherrelation.is_view:
  599.            # for now, assume VIEWS CANNOT BE UPDATED
  600.            return otherrelation
  601.        self.is_shadow = 1
  602.        self.shadow_of_shadow = otherrelation.is_shadow
  603.        self.log = log
  604.        self.name = name
  605.        # don't make any updates permanent if set.
  606.        self.tuples = otherrelation.tuples[:]
  607.        self.attribute_names = otherrelation.attribute_names
  608.        self.filter = otherrelation.filter
  609.        for index in otherrelation.index_list:
  610.            copy = index.copy()
  611.            name = copy.name
  612.            self.add_index(copy, recordtuples=0)
  613.            # record in shadowdb, but don't log it
  614.            inshadowdb.add_index(name, copy)
  615.            #inshadowdb.add_datadef(name, copy, logit=0)
  616.        self.touched = otherrelation.touched
  617.        return self
  618.               
  619.    def unshadow(self):
  620.        """make self into a replacement for shadowed, return self."""
  621.        if self.is_shadow:
  622.            self.log = None
  623.            self.is_shadow = self.shadow_of_shadow
  624.        return self
  625.        
  626.    def dump(self, file):
  627.        attributes = tuple(self.attributes())
  628.        rows = self.rows()
  629.        newrows = rows[:]
  630.        count = 0
  631.        tt = type
  632.        from types import IntType
  633.        for i in xrange(len(rows)):
  634.            this = rows[i]
  635.            if this is not None and tt(this) is not IntType:
  636.               newrows[count] = rows[i].dump(attributes)
  637.               count = count + 1
  638.        newrows = newrows[:count]
  639.        newrows.append(attributes)
  640.        checksum_dump(newrows, file)
  641.  
  642.    def load(self, file):
  643.        """checksum must succeed."""
  644.        rows = checksum_undump(file)
  645.        attributes = rows[-1]
  646.        self.attribute_names = attributes
  647.        rows = rows[:-1]
  648.        from sqlsem import kjbuckets
  649.        undump = kjbuckets.kjUndump
  650.        for i in xrange(len(rows)):
  651.            rows[i] = undump(attributes, rows[i]) 
  652.        self.set_empty()
  653.        self.add_tuples(rows)
  654.        # in sync with disk copy!
  655.        self.touched = 0
  656.        
  657.    def add_index(self, index, recordtuples=1):
  658.        """unset recordtuples if the index is initialized already."""
  659.        # does not "touch" the relation
  660.        index_list = self.index_list
  661.        indices = self.indices
  662.        atts = index.attributes()
  663.        for a in atts:
  664.            indices[a] = index
  665.        if recordtuples:
  666.            (tuples, seqnums) = self.rows(1)
  667.            index.clear()
  668.            if tuples:
  669.                index.add_tuples(tuples, seqnums)
  670.        index_list.append(index)
  671.            
  672.    def drop_index(self, index):
  673.        # does not "touch" the relation
  674.        name = index.name
  675.        if verbosity:
  676.           print "rel.drop_index", index
  677.           print "...", self.indices, self.index_list
  678.        indices = self.indices
  679.        for a in index.attributes():
  680.            # contorted since one index be clone of the other.
  681.            aindices = indices.neighbors(a)
  682.            for ind in aindices:
  683.                if ind.name == name:
  684.                   indices.delete_arc(a, ind)
  685.                   theind = ind
  686.        # the (non-clone) index ought to have been found above...
  687.        self.index_list.remove(theind)
  688.            
  689.    def choose_index(self, attributes):
  690.        """choose an index including subset of attributes or None"""
  691.        from sqlsem import kjbuckets
  692.        kjSet = kjbuckets.kjSet
  693.        atts = kjSet(attributes)
  694.        #print "choosing index", atts
  695.        indices = (atts * self.indices).values()
  696.        choice = None
  697.        for index in indices:
  698.            indexatts = index.attributes()
  699.            #print "index atts", indexatts
  700.            iatts = kjSet(indexatts)
  701.            if iatts.subset(atts):
  702.               if choice is None:
  703.                  #print "chosen", index.name
  704.                  choice = index
  705.                  lchoice = len(choice.attributes())
  706.               else:
  707.                  if index.unique or lchoice<len(indexatts):
  708.                     choice = index
  709.                     lchoice = len(choice.attributes())
  710.        return choice
  711.        
  712.    def __repr__(self):
  713.        rows = self.rows()
  714.        atts = self.attributes()
  715.        list_rep = [list(atts)]
  716.        for r in rows:
  717.            rlist = []
  718.            for a in atts:
  719.                try:
  720.                    elt = r[a]
  721.                except KeyError:
  722.                    elt = "NULL"
  723.                else:
  724.                    elt = str(elt)
  725.                rlist.append(elt)
  726.            list_rep.append(rlist)
  727.        # compute maxen for formatting
  728.        maxen = [0] * len(atts)
  729.        for i in xrange(len(atts)):
  730.            for l in list_rep:
  731.                maxen[i] = max(maxen[i], len(l[i]))
  732.        for i in xrange(len(atts)):
  733.            mm = maxen[i]
  734.            for l in list_rep:
  735.                 old = l[i]
  736.                 l[i] = old + (" " * (mm-len(old)))
  737.        from string import join
  738.        for i in xrange(len(list_rep)):
  739.            list_rep[i] = join(list_rep[i], " | ")
  740.        first = list_rep[0]
  741.        list_rep.insert(1, "=" * len(first))
  742.        return join(list_rep, "\n")
  743.        
  744.    def irepr(self):
  745.        List = [self] + list(self.index_list)
  746.        List = map(str, List)
  747.        from string import join
  748.        return join(List, "\n")
  749.        
  750.    def set_empty(self):
  751.        self.tuples = []
  752.        for index in self.index_list:
  753.            index.clear()
  754.            
  755.    def drop_indices(self, db):
  756.        for index in self.index_list:
  757.            name = index.name
  758.            db.drop_datadef(name)
  759.            db.drop_index(name)
  760.        self.index_list = []
  761.        from sqlsem import kjbuckets
  762.        self.indices = kjbuckets.kjGraph()
  763.            
  764.    def regenerate_indices(self):
  765.        (tuples, seqnums) = self.rows(1)
  766.        #self.tuples = tuples
  767.        for index in self.index_list:
  768.            index.clear()
  769.            index.add_tuples(tuples, seqnums)
  770.        
  771.    def add_tuples(self, tuples):
  772.        if not tuples: return
  773.        tuples = filter(self.filter, tuples)
  774.        oldtuples = self.tuples
  775.        first = len(oldtuples)
  776.        oldtuples[first:] = list(tuples)
  777.        last = len(oldtuples)
  778.        for index in self.index_list:
  779.            index.add_tuples(tuples, xrange(first,last))
  780.        self.touched = 1
  781.            
  782.    def attributes(self):
  783.        return self.attribute_names
  784.        
  785.    def rows(self, andseqnums=0):
  786.        tups = self.tuples
  787.        # short cut
  788.        if 0 not in tups:
  789.           if andseqnums:
  790.              return (tups, xrange(len(tups)))
  791.           else:
  792.              return tups
  793.        tt = type
  794.        from types import IntType
  795.        result = list(self.tuples)
  796.        if andseqnums: seqnums = result[:]
  797.        count = 0
  798.        for i in xrange(len(result)):
  799.            t = result[i]
  800.            if tt(t) is not IntType:
  801.               result[count] = t
  802.               if andseqnums: seqnums[count] = i
  803.               count = count+1
  804.        result = result[:count]
  805.        if andseqnums:
  806.           return (result, seqnums[:count])
  807.        else:
  808.           return result
  809.        
  810.    def erase_tuples(self, seqnums):
  811.        #print "et seqnums", seqnums
  812.        if not seqnums: return
  813.        tups = self.tuples
  814.        # order important! indices first!
  815.        for index in self.index_list:
  816.            index.erase_tuples(seqnums, tups)
  817.        for i in seqnums:
  818.            #print "deleting", i
  819.            tups[i] = 0
  820.        #print self
  821.        self.touched = 1
  822.            
  823.    def reset_tuples(self, tups, seqnums):
  824.        # KISS for indices, maybe optimize someday...
  825.        if not tups: return
  826.        mytups = self.tuples
  827.        for index in self.index_list:
  828.            index.erase_tuples(seqnums, mytups)
  829.        for i in xrange(len(seqnums)):
  830.            seqnum = seqnums[i]
  831.            mytups[seqnum] = tups[i]
  832.        for index in self.index_list:
  833.            index.add_tuples(tups, seqnums)
  834.        self.touched = 1
  835.        
  836. # should views be here?
  837.  
  838. class View(Relation0):
  839.    """view object, acts like relation, with addl operations."""
  840.    touched = 0
  841.    is_view = 1
  842.    is_shadow = 0
  843.    
  844.    ### must fix namelist!
  845.    
  846.    def __init__(self, name, namelist, selection, indb):
  847.        """set namelist to None for implicit namelist"""
  848.        self.name = name
  849.        self.namelist = namelist
  850.        self.selection = selection
  851.        # attempt a relbind, no outer bindings!
  852.        self.relbind(indb, {})
  853.        self.cached_rows = None
  854.        self.translate = None
  855.        
  856.    def __repr__(self):
  857.        return "view %s as %s" % (self.name, self.selection)
  858.        
  859.    irepr = __repr__
  860.        
  861.    def uncache(self):
  862.        self.cached_rows = None
  863.        
  864.    def UNDEFINED_OP_FOR_VIEW(*args, **kw):
  865.        raise ValueError, "operation explicitly undefined for view object"
  866.        
  867.    shadow = dump = load = add_index = drop_index = set_empty = \
  868.    add_tuples = erase_tuples = reset_tuples = UNDEFINED_OP_FOR_VIEW
  869.    
  870.    def ignore_op_for_view(*args, **kw):
  871.        """ignore this op when applied to view"""
  872.        pass
  873.        
  874.    drop_indices = regenerate_indices = ignore_op_for_view
  875.    
  876.    def choose_index(s, a):
  877.        """no indices on views (might change this?)"""
  878.        return None
  879.        
  880.    def relbind(self, db, atts):
  881.        """bind self to db, ignore atts"""
  882.        name = self.name
  883.        selection = self.selection
  884.        selection = self.selection = selection.relbind(db)
  885.        namelist = self.namelist
  886.        if namelist is not None:
  887.            from sqlsem import kjbuckets
  888.            target_atts = selection.attributes()
  889.            if len(namelist)!=len(target_atts):
  890.               raise "select list and namelist don't match in %s"%name
  891.            pairs = map(None, namelist, target_atts)
  892.            self.translate = kjbuckets.kjGraph(pairs)
  893.        return self
  894.        
  895.    def attributes(self):
  896.        namelist = self.namelist
  897.        if self.namelist is None:
  898.            return self.selection.attributes()
  899.        return namelist
  900.        
  901.    def rows(self, andseqs=0):
  902.        cached_rows = self.cached_rows
  903.        if cached_rows is None:
  904.           cached_rows = self.cached_rows = self.selection.eval().rows()
  905.           if self.namelist is not None:
  906.               # translate the attribute names
  907.               translate = self.translate
  908.               for i in range(len(cached_rows)):
  909.                   cached_rows[i] = cached_rows[i].remap(translate)
  910.        if andseqs:
  911.           return (cached_rows[:], range(len(cached_rows)))
  912.        else:
  913.           return cached_rows[:]
  914.        
  915. class Index:
  916.    """Index for tuples in relation.  Tightly bound to relation rep."""
  917.    
  918.    ### should add "unique index" and check enforce uniqueness...
  919.    
  920.    def __init__(self, name, attributes, unique=0):
  921.        self.unique = unique
  922.        self.name = name
  923.        self.atts = tuple(attributes)
  924.        # values > tuples
  925.        self.index = {}
  926.        self.dseqnums = {}
  927.        
  928.    def __repr__(self):
  929.        un = ""
  930.        if self.unique: un="UNIQUE "
  931.        return "%sindex %s on %s" % (un, self.name, self.atts)
  932.        
  933.    def copy(self):
  934.        """make a fast structural copy of self"""
  935.        result = Index(self.name, self.atts, unique=self.unique)
  936.        rindex = result.index
  937.        rdseqnums = result.dseqnums
  938.        myindex = self.index
  939.        mydseqnums = self.dseqnums
  940.        for k in myindex.keys():
  941.            rindex[k] = myindex[k][:]
  942.        for k in mydseqnums.keys():
  943.            rdseqnums[k] = mydseqnums[k][:]
  944.        return result
  945.        
  946.    def attributes(self):
  947.        return self.atts
  948.        
  949.    def matches(self, tuple, translate=None):
  950.        """return (tuples, seqnums) for tuples matching tuple
  951.           (with possible translations"""
  952.        if translate:
  953.           tuple = translate * tuple
  954.        atts = self.atts
  955.        dump = tuple.dump(atts)
  956.        index = self.index
  957.        if index.has_key(dump):
  958.           return (index[dump], self.dseqnums[dump])
  959.        else:
  960.           return ((), ())
  961.        
  962.    def clear(self):
  963.        self.index = {}
  964.        self.dseqnums = {}
  965.        
  966.    def add_tuples(self, tuples, seqnums):
  967.        unique = self.unique
  968.        atts = self.atts
  969.        index = self.index
  970.        dseqnums = self.dseqnums
  971.        test = index.has_key
  972.        for i in xrange(len(tuples)):
  973.            tup = tuples[i]
  974.            seqnum = seqnums[i]
  975.            dump = tup.dump(atts)
  976.            #print self.name, dump
  977.            if test(dump):
  978.               bucket = index[dump]
  979.               #print "self", self
  980.               #print "unique", unique
  981.               #print "bucket", bucket
  982.               if unique and bucket:
  983.                   raise StorageError, "uniqueness violation: %s %s" %(
  984.                     dump, self)
  985.               bucket.append(tup)
  986.               dseqnums[dump].append(seqnum)
  987.            else:
  988.               index[dump] = [tup]
  989.               dseqnums[dump] = [seqnum]
  990.            
  991.    def erase_tuples(self, seqnums, all_tuples):
  992.        # all_tuples must be internal rel tuple list
  993.        atts = self.atts
  994.        index = self.index
  995.        dseqnums = self.dseqnums
  996.        for seqnum in seqnums:
  997.            tup = all_tuples[seqnum]
  998.            dump = tup.dump(atts)
  999.            index[dump].remove(tup)
  1000.            dseqnums[dump].remove(seqnum)
  1001.            
  1002. class shadow_dict:
  1003.     """shadow dictionary. defer & remember updates."""
  1004.     verbose = verbosity
  1005.     def __init__(self, shadowing, value_transform=None):
  1006.         self.shadowed = shadowing
  1007.         shadow = self.shadow = {}
  1008.         self.touched = {}
  1009.         for key in shadowing.keys():
  1010.             shadow[key] = shadowing[key]
  1011.         self.value_transform = value_transform
  1012.         # defeats inheritance! careful!
  1013.         self.values = shadow.values
  1014.         self.items = shadow.items
  1015.         self.keys = shadow.keys
  1016.         self.has_key = shadow.has_key
  1017.         
  1018.     def is_shadowed(self, name):
  1019.         return self.touched.has_key(name)
  1020.         
  1021.     def __len__(self):
  1022.         return len(self.shadow)
  1023.         
  1024.     def commit(self, verbose=0):
  1025.         """apply updates to shadowed."""
  1026.         import sys
  1027.         verbose = verbose or self.verbose
  1028.         if self.touched:
  1029.             shadowed = self.shadowed
  1030.             shadow = self.shadow
  1031.             value_transform = self.value_transform
  1032.             keys = shadowed.keys()
  1033.             if verbose:
  1034.                 print "shadowdict oldkeys", keys
  1035.             for k in keys:
  1036.                 del shadowed[k]
  1037.             keys = shadow.keys()
  1038.             if verbose:
  1039.                 print "shadowdict newkeys", keys
  1040.             for k in shadow.keys():
  1041.                 value = shadow[k]
  1042.                 if value_transform is not None:
  1043.                    try:
  1044.                        value = value_transform(value)
  1045.                    except:
  1046.                        raise "transform fails", (sys.exc_type, sys.exc_value, k, value)
  1047.                 shadowed[k] = value
  1048.             self.touched = {}
  1049.                     
  1050.     def __getitem__(self, key):
  1051.         return self.shadow[key]
  1052.            
  1053.     def __setitem__(self, key, item):
  1054.         from types import StringType
  1055.         if type(key) is not StringType:
  1056.            raise "nonstring", key
  1057.         if item is None:
  1058.            raise "none set", (key, item)
  1059.         self.touched[key] = 1
  1060.         self.shadow[key] = item
  1061.         
  1062.     def __delitem__(self, key):
  1063.         self.touched[key] = 1
  1064.         del self.shadow[key]
  1065.         
  1066. # stored mutations on relations
  1067. class Add_Tuples:
  1068.    """stored rel.add_tuples(tuples)"""
  1069.    def __init__(self, name):
  1070.        self.to_rel = name
  1071.        self.indb = None
  1072.    def initargs(self):
  1073.        return (self.to_rel,)
  1074.    def set_data(self, tuples, rel):
  1075.        """store self.data as tuple with tuple[-1] as to_rel, rest data"""
  1076.        attributes = tuple(rel.attributes())
  1077.        ltuples = len(tuples)
  1078.        data = list(tuples)
  1079.        for i in xrange(ltuples):
  1080.            tdata = tuples[i].dump(attributes)
  1081.            data[i] = tdata
  1082.        self.data = tuple(data)
  1083.    def __repr__(self):
  1084.        from string import join
  1085.        datarep = map(repr, self.data)
  1086.        datarep = join(datarep, "\n  ")
  1087.        return "add tuples to %s\n  %s\n\n" % (self.to_rel, datarep)
  1088.    def marshaldata(self):
  1089.        return self.data
  1090.    def demarshal(self, data):
  1091.        self.data = data
  1092.    def relbind(self, db):
  1093.        self.indb = db
  1094.    def eval(self, dyn=None):
  1095.        """apply operation to db"""
  1096.        db = self.indb
  1097.        data = self.data
  1098.        name = self.to_rel
  1099.        rel = db[name]
  1100.        attributes = tuple(rel.attributes())
  1101.        tuples = list(data)
  1102.        from sqlsem import kjbuckets
  1103.        undump = kjbuckets.kjUndump
  1104.        for i in xrange(len(tuples)):
  1105.            tuples[i] = undump(attributes, tuples[i])
  1106.        rel.add_tuples(tuples)
  1107.        
  1108. class Erase_Tuples(Add_Tuples):
  1109.    """stored rel.erase_tuples(seqnums)"""
  1110.    def set_data(self, seqnums, rel):
  1111.        seqnums = list(seqnums)
  1112.        self.data = tuple(seqnums)
  1113.    def __repr__(self):
  1114.        return "Erase seqnums in %s\n  %s\n\n" % (self.to_rel, self.data)
  1115.    def eval(self, dyn=None):
  1116.        db = self.indb
  1117.        seqnums = self.data
  1118.        name = self.to_rel
  1119.        rel = db[name]
  1120.        rel.erase_tuples(seqnums)
  1121.        
  1122. class Reset_Tuples(Add_Tuples):
  1123.    """stored rel.reset_tuples(tups, seqnums)"""
  1124.    def set_data(self, tups, seqnums, rel):
  1125.        attributes = tuple(rel.attributes())
  1126.        dtups = list(tups)
  1127.        for i in xrange(len(dtups)):
  1128.            dtups[i] = dtups[i].dump(attributes)
  1129.        self.data = (tuple(dtups), tuple(seqnums))
  1130.    def __repr__(self):
  1131.        (dtups, seqnums) = self.data
  1132.        pairs = map(None, seqnums, dtups)
  1133.        from string import join
  1134.        datarep = map(repr, pairs)
  1135.        datarep = join(datarep, "  \n")
  1136.        return "Reset tuples in %s\n  %s\n\n" % (self.to_rel, datarep)
  1137.    def eval(self, dyn=None):
  1138.        db = self.indb
  1139.        (dtups, seqnums) = self.data
  1140.        tups = list(dtups)
  1141.        rel = db[self.to_rel]
  1142.        attributes = tuple(rel.attributes())
  1143.        from sqlsem import kjbuckets
  1144.        undump = kjbuckets.kjUndump
  1145.        for i in xrange(len(dtups)):
  1146.            tups[i] = undump(attributes, dtups[i])
  1147.        rel.reset_tuples(tups, seqnums)
  1148.         
  1149. # Log entry tags
  1150. START = "START"
  1151. COMMIT = "COMMIT"
  1152. ABORT = "ABORT"
  1153. UNREADABLE = "UNREADABLE"
  1154.         
  1155. class Transaction_Logger:
  1156.    """quick and dirty Log implementation per transaction."""
  1157.    verbose = verbosity
  1158.    
  1159.    def __init__(self, db_log, transactionid, is_scratch=0):
  1160.        self.db_log = db_log
  1161.        self.transactionid = transactionid
  1162.        # ignore all operations if set
  1163.        self.is_scratch = is_scratch
  1164.        self.dirty = 0
  1165.        self.deferred = []
  1166.        
  1167.    def reset(self):
  1168.        self.deferred = []
  1169.        
  1170.    def __repr__(self):
  1171.        return "Transaction_Logger(%s, %s, %s)" % (
  1172.           self.db_log, self.transactionid, self.is_scratch)
  1173.        
  1174.    def log(self, operation):
  1175.        verbose = self.verbose
  1176.        tid = self.transactionid
  1177.        if not self.is_scratch:
  1178.           self.deferred.append(operation)
  1179.           if verbose:
  1180.              print "tid logs", tid, operation
  1181.              
  1182.    def flush(self):
  1183.        verbose = self.verbose
  1184.        if not self.is_scratch:
  1185.           tid = self.transactionid
  1186.           deferred = self.deferred
  1187.           self.deferred = []
  1188.           db_log = self.db_log
  1189.           if db_log:
  1190.               for operation in deferred:
  1191.                   db_log.log(operation, tid)
  1192.           self.dirty = 1
  1193.        elif verbose:
  1194.           print "scratch log ignored", tid, operation
  1195.        
  1196.    def commit(self, verbose=0):
  1197.        verbose = self.verbose or verbose
  1198.        tid = self.transactionid
  1199.        if verbose: print "committing trans log", tid
  1200.        if self.is_scratch: 
  1201.            if verbose:
  1202.                print "scratch commit ignored", tid
  1203.            return
  1204.        if not self.dirty:
  1205.            if verbose:
  1206.                print "nondirty commit", tid
  1207.            return
  1208.        self.flush()
  1209.        db_log = self.db_log
  1210.        db_log.commit(verbose, tid)
  1211.        if verbose:
  1212.           print "transaction is considered recoverable", tid
  1213.           
  1214. class DB_Logger:
  1215.    """quick and dirty global db logger."""
  1216.    verbose = verbosity
  1217.    is_scratch = 0
  1218.    
  1219.    def __init__(self, filename, backupname):
  1220.        self.filename = filename
  1221.        # backup name is never kept open: existence indicates log in use.
  1222.        self.backupname = backupname
  1223.        self.file = None
  1224.        self.dirty = 0
  1225.        if self.verbose:
  1226.           print id(self), "created DB_Logger on", self.filename
  1227.           
  1228.    def __repr__(self):
  1229.        return "DB_Logger(%s)" % self.filename
  1230.        
  1231.    def startup(self):
  1232.        if self.verbose:
  1233.            print id(self), "preparing", self.filename
  1234.        # open happens automagically
  1235.        #self.file = open(self.filename, "wb")
  1236.        self.clear()
  1237.        self.dirty = 0
  1238.        
  1239.    def shutdown(self):
  1240.        if self.verbose:
  1241.            print id(self), "shutting down log", self.filename
  1242.        file = self.file
  1243.        if file:
  1244.            file.close()
  1245.        self.file = None
  1246.        
  1247.    def clear(self):
  1248.        if self.verbose:
  1249.            print id(self), "clearing"
  1250.        self.shutdown()
  1251.        del_file(self.filename)
  1252.        
  1253.    def restart(self):
  1254.        if self.verbose:
  1255.            print id(self), "restarting log file", self.filename
  1256.        if self.file is not None:
  1257.           self.file.close()
  1258.        self.file = open(self.filename, "ab")
  1259.        dummy = open(self.backupname, "ab")
  1260.        dummy.close()
  1261.        self.dirty = 0
  1262.        
  1263.    def clear_log_file(self):
  1264.        if self.verbose:
  1265.            print id(self), "clearing logfile", self.filename
  1266.        if self.file is not None:
  1267.           self.file.close()
  1268.           self.file = None
  1269.        del_file(self.filename)
  1270.        del_file(self.backupname)
  1271.        self.dirty = 0
  1272.        
  1273.    def log(self, operation, transactionid=None):
  1274.        """transactionid of None means no transaction: immediate."""
  1275.        file = self.file
  1276.        if file is None:
  1277.           self.restart()
  1278.           file = self.file
  1279.        verbose = self.verbose
  1280.        from sqlsem import serialize
  1281.        serial = serialize(operation)
  1282.        data = (transactionid, serial)
  1283.        if verbose:
  1284.           print id(self), "logging:", transactionid
  1285.           print operation
  1286.        checksum_dump(data, file)
  1287.        self.dirty = 1
  1288.        
  1289.    def commit(self, verbose=0, transactionid=None):
  1290.        """add commit, if appropriate, flush."""
  1291.        verbose = self.verbose or verbose
  1292.        if not self.dirty and transactionid is None:
  1293.            if verbose: print "commit not needed", transactionid
  1294.            return
  1295.        elif verbose:
  1296.            print "attempting commit", transactionid
  1297.        if transactionid is not None:
  1298.           self.log( COMMIT, transactionid )
  1299.           if verbose: print "committed", transactionid
  1300.        if verbose: print "flushing", self.filename
  1301.        self.file.flush()
  1302.        self.dirty = 0
  1303.        
  1304.    def recover(self, db, restart=1):
  1305.        import sys
  1306.        verbose = self.verbose
  1307.        filename = self.filename
  1308.        if verbose:
  1309.           print "attempting recovery from", self.filename
  1310.        file = self.file
  1311.        if file is not None:
  1312.           if verbose: print "closing file"
  1313.           self.file.close()
  1314.           self.file = None
  1315.        if verbose:
  1316.            print "opens should generate an error if no recovery needed"
  1317.        try:
  1318.            file = open(filename, "rb")
  1319.            file2 = open(self.backupname, "rb")
  1320.        except:
  1321.            if verbose: 
  1322.                print "no recovery needed:", filename
  1323.                print sys.exc_type, sys.exc_value
  1324.            sys.exc_traceback = None
  1325.            return
  1326.        file2.close()
  1327.        if verbose: print "log found, recovering from", filename
  1328.        records = self.read_records(file)
  1329.        if verbose: print "scan for commit records"
  1330.        commits = {}
  1331.        for (i, (tid, op)) in records:
  1332.            if op==COMMIT:
  1333.                if verbose: print "transaction", tid, "commit at", i
  1334.                commits[tid] = i
  1335.            elif verbose:
  1336.                print i, tid, "operation\n", op
  1337.        if verbose: print commits, "commits total"
  1338.        if verbose: print "applying commited operations, in order"
  1339.        committed = commits.has_key
  1340.        from types import StringType
  1341.        for (i, (tid, op)) in records:
  1342.            if tid is None or (committed(tid) and commits[tid]>i):
  1343.                if type(op) is StringType:
  1344.                    if verbose:
  1345.                        print "skipping marker", tid, op
  1346.                if verbose:
  1347.                    print "executing for", tid, i
  1348.                    print op
  1349.                #### Note: silently eat errors unless verbose
  1350.                ### (eg in case of table recreation...)
  1351.                ### There should be a better way to do this!!!
  1352.                import sys
  1353.                try:
  1354.                    op.relbind(db)
  1355.                    op.eval()
  1356.                except:
  1357.                    if verbose:
  1358.                        print "error", sys.exc_type, sys.exc_value
  1359.                        print "binding or evaluating logged operation:"
  1360.                        print op
  1361.            elif verbose:
  1362.                print "uncommitted operation", tid, i
  1363.                op
  1364.        if verbose:
  1365.           print "recovery successful: clearing log file"
  1366.        self.clear()
  1367.        if restart:
  1368.            if verbose:
  1369.                print "recreating empty log file"
  1370.            self.startup()
  1371.                
  1372.    def read_records(self, file):
  1373.        """return log record as (index, (tid, op)) list"""
  1374.        verbose = self.verbose
  1375.        if verbose: print "reading log records to error"
  1376.        import sys
  1377.        records = {}
  1378.        from sqlsem import deserialize
  1379.        count = 0
  1380.        while 1:
  1381.            try:
  1382.                data = checksum_undump(file)
  1383.            except:
  1384.                if verbose:
  1385.                   print "record read terminated with error", len(records)
  1386.                   print sys.exc_type, sys.exc_value
  1387.                break
  1388.            (transactionid, serial) = data
  1389.            operation = deserialize(serial)
  1390.            records[count] = (transactionid, operation)
  1391.            if verbose:
  1392.                print count, ": read for", transactionid
  1393.                print operation
  1394.            count = count+1
  1395.        if verbose: print len(records), "records total"
  1396.        records = records.items()
  1397.        records.sort()
  1398.        return records
  1399.        
  1400.    def dump(self):
  1401.        verbose = self.verbose
  1402.        self.shutdown()
  1403.        print "dumping log"
  1404.        self.verbose = 1
  1405.        try:
  1406.            file = open(self.filename, "rb")
  1407.        except:
  1408.            print "DUMP FAILED, cannot open", self.filename
  1409.        else:
  1410.            self.read_records(file)
  1411.        self.verbose = verbose
  1412.        self.restart()
  1413.